import scrapy

from db_tushu.items import DbTushuItem


class TushuSpiderSpider(scrapy.Spider):
    name = 'tushu_spider'
    allowed_domains = ['book.douban.com']
    # start_urls = ['https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4']
    books_url = []
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36'
    }
    start = 200

    def start_requests(self):
        url = 'https://book.douban.com/tag/%E9%9A%8F%E7%AC%94?start=' + str(self.start) + '&type=T'
        yield scrapy.Request(url=url, headers=self.headers, callback=self.parse)

    def parse(self, response):
        self.books_url = response.xpath('//div[@class="pic"]/a/@href').getall()
        # book_url = 'https://book.douban.com/subject/6953273/'
        for book_url in self.books_url:
            yield scrapy.Request(url=book_url, headers=self.headers, callback=self.parse2)

        # if self.books_url != None:
        for start in range(1, 3):    # 不能用for，因为下一次调用该函数的时候会再次启动for，进入无限
            self.start = self.start + start * 20
            if self.start == 240:
                return
            next_page = 'https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4?start=' + str(self.start) + '&type=T'
            yield response.follow(next_page, headers=self.headers, callback=self.parse)

    def parse2(self, response):
        item = DbTushuItem()
        item['title'] = response.xpath('//h1/span/text()').get()
        author = response.xpath('//div[@id="info"]').re_first(r'<a.*">((?:.|\n)*?)</a>')
        item['author'] = remove_spaces(author)
        item['binding'] = response.xpath('//div[@id="info"]').re_first(r'装帧:</span>(.*?)<br').strip(' ')
        item['publisher'] = response.xpath('//div[@id="info"]').re_first(r'出版社:</span>(.*?)<br').strip(' ')
        item['price'] = response.xpath('//div[@id="info"]').re_first(r'定价:</span>(.*?)<br').strip(' ')
        item['pages'] = response.xpath('//div[@id="info"]').re_first(r'页数:</span>(.*?)<br')
        item['pubdate'] = response.xpath('//div[@id="info"]').re_first(r'出版年:</span>(.*?)<br').strip(' ')
        item['isbn'] = response.xpath('//div[@id="info"]').re_first(r'ISBN:</span>(.*?)<br').strip(' ')
        print(len(response.xpath('//div[@class="intro"]').getall()))
        item['summary'] = response.xpath('//span[@class="all hidden"]/div/div').re_first(r'(<p>.*</p>)')
        if item['summary'] == None:
            item['summary'] = response.xpath('//div[@class="intro"][1]').re_first(r'(<p>.*</p>)')
        item['image'] = response.xpath('//a[@class="nbg"]/img/@src').get()
        # print(item['title'], item['author'], item['binding'], item['publisher'], item['price'], item['pages'],
        #       item['pubdate'], item['isbn'], item['summary'], item['image'])
        yield item


def remove_spaces(title):
    title = title.replace(' ', '').replace('\n', '').replace('\r', '')
    return title
